package edu.fudan.cse.medlab.event.filter;

import java.io.BufferedReader;
import java.io.FileReader;

public class CrawlWebList extends Thread {
	Settings setting;
	long[] process;
	String[] urllist;
	String listfile;
	String dir;
	public CrawlWebList(Settings setting,String listfile,String dir){
		this.setting = setting;
		this.listfile = listfile;
		this.dir = dir;
	}
	
	public void run(){
		try{
			startCrawl(this.listfile,this.dir);
			updateProcess();
		}
		catch(Exception e){
			e.printStackTrace();
		}
	}
	public void startCrawl(String listfile,String dir) throws Exception{
	
		BufferedReader bin = new BufferedReader(new FileReader(listfile));
		String url,s="";
		while((url = bin.readLine()) != null){
			if(url.contains(" "))
				url = url.substring(0,url.indexOf(" "));
			if(url.contains("\t"))
				url = url.substring(0,url.indexOf("\t"));
			url.trim();
			s+=url+"|";
		}
		if(s.endsWith("|"))
			urllist = s.split("\\|");
		else
			return;
		WGet wget[] = new WGet[urllist.length];
		for (int i = 0; i < wget.length; i++) {
			wget[i] = new WGet(urllist[i],dir);
			wget[i].start();
			edu.fudan.cse.medlab.event.extraction.DB.addLog("start crawling site:"+urllist[i]);
			sleep(30*1000);
		}		
		//System.out.println("Crawl start at:"+new Time(System.currentTimeMillis()).toString());	
	}
	
	public void updateProcess() throws Exception{
		int t = 60*1000*setting.updatefrequent;
		boolean finished = false;
		//DB.open(setting.host, setting.dbname, setting.username,setting.password);
		process = new long[urllist.length];
		for (int i = 0; i < process.length; i++) {
			process[i] = -1;
		}
		while(true){
			finished = true;
			int s=1;
			edu.fudan.cse.medlab.event.extraction.DB.addLog("Crawler state updated");
			for (int i = 0; i < urllist.length; i++) {
				long k = FileSize.checkDirSize(dir+"\\"+urllist[i]);
				edu.fudan.cse.medlab.event.extraction.DB.addLog("Crawling "+urllist[i]+"    downloaded "+k+"Bytes");
				if(k != process[i]){
					process[i] = k;
					finished = false;
				}
			}
			if(finished)
				break;
			else
				sleep(t);
		}
		edu.fudan.cse.medlab.event.extraction.DB.addLog("Crawler finished");
		DataBase.setState(DataBase.getState()+1);
	}
	
	public static void main(String[] args){
		String listfile ="e:\\jmj\\weblist.txt";
		CrawlWebList c = new CrawlWebList(new Settings(),listfile,"seedsite");
		c.start();
	}
}
